#!/bin/bash

baseDirForScriptSelf=$(cd "$(dirname "$0")"; pwd)
PWDNOW=${baseDirForScriptSelf}

eval $(grep logPath ${PWDNOW}/tag.cfg) #

eval $(grep HIVE_DATABASE ${PWDNOW}/tag.cfg) #

eval $(grep TBNAME_JINRONG ${PWDNOW}/tag.cfg) #
eval $(grep TBNAME_FANGCHAN ${PWDNOW}/tag.cfg) #
eval $(grep TBNAME_AREAHOUSE ${PWDNOW}/tag.cfg) #
eval $(grep TBNAME_WEIXIN ${PWDNOW}/tag.cfg) #

eval $(grep TBNAME_TAG_4G ${PWDNOW}/tag.cfg) #
eval $(grep jinrong_rule_Path ${PWDNOW}/tag.cfg) #
eval $(grep fangchan_rule_Path ${PWDNOW}/tag.cfg) #
eval $(grep areahouse_rule_Path ${PWDNOW}/tag.cfg) #
eval $(grep weixin_rule_Path ${PWDNOW}/tag.cfg) #


eval $(grep TAG_START_DATE ${PWDNOW}/tag.cfg) #
eval $(grep TAG_STOP_DATE ${PWDNOW}/tag.cfg) #

eval $(grep jinrong_TOPN ${PWDNOW}/tag.cfg) #
eval $(grep fangchan_TOPN ${PWDNOW}/tag.cfg) #
eval $(grep areahouse_TOPN ${PWDNOW}/tag.cfg) #
eval $(grep weixin_TOPN ${PWDNOW}/tag.cfg) #


eval $(grep RESULT_PATH ${PWDNOW}/tag.cfg) #
eval $(grep TAG_4G_RESULT ${PWDNOW}/tag.cfg) #



DAY=`date "+%Y%m%d"`
logTask=$logPath/RuleTask/${DAY}
logAnalyse=$logPath/RuleAnalyse/${DAY}

if [ ! -d "$logTask" ]; then
        mkdir -p $logTask
        chmod 755 $logTask
fi

if [ ! -d "$logAnalyse" ]; then
        mkdir -p $logAnalyse
        chmod 755 $logAnalyse
fi

DATE=`date "+%Y%m%d%H%M%S"`

if [ -f "${PWDNOW}/run_once" ]; then
	echo "run nothing at ${DATE}!" >>$logPath/RuleAnalyse/Rule_users.log
	exit 0
fi

touch ${PWDNOW}/run_once

hadoop fs -rm -r $RESULT_PATH
hadoop fs -mkdir -p $RESULT_PATH


nohup hive -e "

use ${HIVE_DATABASE};
set mapred.max.map.failures.percent=100;
set mapred.max.reduce.failures.percent=100;
set hive.exec.reducers.max=200;
set hive.exec.max.created.files=655350;
set mapred.max.split.size=268435456;
set mapred.min.split.size.per.node=134217728;
set mapred.min.split.size.per.rack=134217728;
set hive.hadoop.supports.splittable.combineinputformat=true;
set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
set hive.merge.mapfiles=true;
set hive.merge.mapredfiles=true;
set hive.merge.size.per.task=268435456;
set hive.merge.smallfiles.avgsize=134217728;
set hive.auto.convert.join=false;
set hive.groupby.skewindata=true;
set hive.exec.dynamic.partition=true;
set hive.exec.dynamic.partition.mode=nonstrict;


DROP TABLE ${TBNAME_JINRONG};
CREATE EXTERNAL TABLE IF NOT EXISTS ${HIVE_DATABASE}.${TBNAME_JINRONG}(
id       string comment 'id',
rule_type     string comment '规则类型',
host    string comment '域名',
rule_name1   string comment '参数',
rule_name2    string comment '参数',
type_code	string comment '标签编码'
) 
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\|';

load data inpath '${jinrong_rule_Path}' OVERWRITE into table ${HIVE_DATABASE}.${TBNAME_JINRONG};


DROP TABLE ${TBNAME_FANGCHAN};
CREATE EXTERNAL TABLE IF NOT EXISTS ${HIVE_DATABASE}.${TBNAME_FANGCHAN}(
id       string comment 'id',
rule_type     string comment '规则类型',
host    string comment '域名',
rule_name1   string comment '参数',
rule_name2    string comment '参数',
type_code	string comment '标签编码'
) 
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\|';

load data inpath '${fangchan_rule_Path}' OVERWRITE into table ${HIVE_DATABASE}.${TBNAME_FANGCHAN};

DROP TABLE ${TBNAME_AREAHOUSE};
CREATE EXTERNAL TABLE IF NOT EXISTS ${HIVE_DATABASE}.${TBNAME_AREAHOUSE}(
id       string comment 'id',
rule_type     string comment '规则类型',
host    string comment '域名',
rule_name1   string comment '参数',
rule_name2    string comment '参数',
type_code	string comment '标签编码'
) 
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\|';

load data inpath '${areahouse_rule_Path}' OVERWRITE into table ${HIVE_DATABASE}.${TBNAME_AREAHOUSE};


DROP TABLE ${TBNAME_WEIXIN};
CREATE EXTERNAL TABLE IF NOT EXISTS ${HIVE_DATABASE}.${TBNAME_WEIXIN}(
id       string comment 'id',
rule_type     string comment '规则类型',
host    string comment '域名',
rule_name1   string comment '参数',
rule_name2    string comment '参数',
type_code	string comment '标签编码'
) 
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\|';

load data inpath '${weixin_rule_Path}' OVERWRITE into table ${HIVE_DATABASE}.${TBNAME_WEIXIN};



DROP TABLE ${TAG_4G_RESULT};
CREATE EXTERNAL TABLE IF NOT EXISTS ${HIVE_DATABASE}.${TAG_4G_RESULT}(
host    string comment '域名',
rule_name1   string comment '参数',
rule_name2    string comment '参数',
type_code	string comment '标签编码'
) 
PARTITIONED BY(data_type string comment '数据来源类型 3G/4G',datetime string comment '日期')
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\|';

insert overwrite table ${HIVE_DATABASE}.${TAG_4G_RESULT} partition(data_type='4g',datetime='${DAY}')  

   select t.host,t.rule_name1,t.rule_name2,t.type_code from  ${HIVE_DATABASE}.${TBNAME_JINRONG} t join 
       (
       select count(a.jinrong) sumtag,a.jinrong  from 
           
                ${HIVE_DATABASE}.${TBNAME_TAG_4G} a  
                
       where a.jinrong!='' and a.datetime>='${TAG_START_DATE}' 
             and a.datetime<='${TAG_STOP_DATE}'  
       group by a.jinrong        
       order by sumtag desc 
       limit ${jinrong_TOPN} 
       ) s on t.type_code = s.jinrong;

	   
insert into table ${HIVE_DATABASE}.${TAG_4G_RESULT} partition(data_type='4g',datetime='${DAY}')  

   select t.host,t.rule_name1,t.rule_name2,t.type_code from  ${HIVE_DATABASE}.${TBNAME_FANGCHAN} t join 
   (
   select fangchan from
	(
	select sumtag,fangchan from 
       (
       select count(a.fangchan) sumtag,a.fangchan  from 
           
                ${HIVE_DATABASE}.${TBNAME_TAG_4G} a  
                
       where a.fangchan!='' and a.datetime>='${TAG_START_DATE}' 
             and a.datetime<='${TAG_STOP_DATE}'  
       group by a.fangchan        
       order by sumtag desc 
       limit ${fangchan_TOPN} 
       ) tt LATERAL VIEW explode(split(tt.fangchan,',')) tt as fangchan   
	) tmp  group by tmp.fangchan 
  ) s	on t.type_code = s.fangchan;	 

insert into table ${HIVE_DATABASE}.${TAG_4G_RESULT} partition(data_type='4g',datetime='${DAY}')  

   select t.host,t.rule_name1,t.rule_name2,t.type_code from  ${HIVE_DATABASE}.${TBNAME_AREAHOUSE} t join 
       (
       select count(a.quyufangchan) sumtag,a.quyufangchan  from 
           
                ${HIVE_DATABASE}.${TBNAME_TAG_4G} a  
                
       where a.quyufangchan!='' and a.datetime>='${TAG_START_DATE}' 
             and a.datetime<='${TAG_STOP_DATE}'  
       group by a.quyufangchan        
       order by sumtag desc 
       limit ${areahouse_TOPN} 
       ) s on t.type_code = s.quyufangchan;	   

insert into table ${HIVE_DATABASE}.${TAG_4G_RESULT} partition(data_type='4g',datetime='${DAY}')  

   select t.host,t.rule_name1,t.rule_name2,t.type_code from  ${HIVE_DATABASE}.${TBNAME_WEIXIN} t join 
       (
       select count(a.weixin) sumtag,a.weixin  from 
           
                ${HIVE_DATABASE}.${TBNAME_TAG_4G} a  
                
       where a.weixin!='' and a.datetime>='${TAG_START_DATE}' 
             and a.datetime<='${TAG_STOP_DATE}'  
       group by a.weixin        
       order by sumtag desc 
       limit ${weixin_TOPN} 
       ) s on t.type_code = s.weixin;	
	  
INSERT OVERWRITE  DIRECTORY '${RESULT_PATH}'   
select host, 
       rule_name1,
       rule_name2,  
       type_code           
from ${HIVE_DATABASE}.${TAG_4G_RESULT};
	  
">>$logAnalyse/rule_match.log 2>&1 &                                                                            

